{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GRU in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we use a GRU to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n",
    "from keras.layers import GRU # new! \n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/gru'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64 \n",
    "n_unique_words = 10000 \n",
    "max_review_length = 100 # lowered due to vanishing gradient over time\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# GRU layer architecture:\n",
    "n_gru = 256 \n",
    "drop_gru = 0.2\n",
    "\n",
    "# dense layer architecture: \n",
    "# n_dense = 256\n",
    "# dropout = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
    "model.add(SpatialDropout1D(drop_embed))\n",
    "model.add(GRU(n_gru, dropout=drop_gru))\n",
    "# model.add(Dense(n_dense, activation='relu')) # typically don't see top dense layer in NLP like in \n",
    "# model.add(Dropout(dropout))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 100, 64)           640000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 100, 64)           0         \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 256)               246528    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 886,785\n",
      "Trainable params: 886,785\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 31s - loss: 0.5238 - acc: 0.7241 - val_loss: 0.3771 - val_acc: 0.8350\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 29s - loss: 0.3062 - acc: 0.8731 - val_loss: 0.3390 - val_acc: 0.8504\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 29s - loss: 0.2484 - acc: 0.9004 - val_loss: 0.3680 - val_acc: 0.8452\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 28s - loss: 0.2123 - acc: 0.9176 - val_loss: 0.3837 - val_acc: 0.8396\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd996b5e438>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# XXX% validation accuracy in epoch X\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 24s    \n"
     ]
    }
   ],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE5NJREFUeJzt3X+s3fV93/HnKzgka5vEJlwQss1MVDcLrRTCLCCK1NE4\nM8apMH+EydE6XGTNU0erdqu2kO0Pb9BMZNPGhtTSucWridoQypZhJazMc0DZpplgCqUBinxDKFyZ\nYReDsw4lHel7f5yPkwO51/dc+95zcvk8H9LR9/t9fz/f8/18fO37Ot9fx6kqJEn9edukOyBJmgwD\nQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpFZPuwKmce+65tW7dukl3Q/pB33pm\nMH33+yfbD2kWjz766J9V1dR87X6oA2DdunUcOnRo0t2QftB/u3Iw/dhDk+yFNKskfzpKO08BSVKn\nDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqXkDIMn7kzw+9PpWkl9Jck6S/UkOt+mq1j5Jbk8y\nneSJJJcOvdf21v5wku1LOTBJ0qnNGwBV9UxVXVJVlwB/HXgN+CJwE3CgqtYDB9oywNXA+vbaCdwB\nkOQcYBdwOXAZsOtkaEiSxm+hTwJvBL5RVX+aZCtwZavvBR4CPgVsBe6qwf82fzDJyiQXtLb7q+o4\nQJL9wGbg82c6CElaCutu+vLE9v3crR9f8n0s9BrANr7/C/v8qnoRoE3Pa/XVwAtD28y02lz1N0iy\nM8mhJIeOHTu2wO5JkkY1cgAkORu4Bvj9+ZrOUqtT1N9YqNpdVRuqasPU1LzfZSRJOk0LOQK4GvjD\nqnqpLb/UTu3QpkdbfQZYO7TdGuDIKeqSpAlYSAB8kjeer98HnLyTZztw31D9+nY30BXAiXaK6AFg\nU5JV7eLvplaTJE3ASBeBk/wI8DeBvzdUvhW4J8kO4Hngula/H9gCTDO4Y+gGgKo6nuQW4JHW7uaT\nF4QlSeM3UgBU1WvAe99Ue5nBXUFvblvAjXO8zx5gz8K7eXomdQV/HFfvJelM+SSwJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJ\nnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6NVIAJFmZ5N4kf5Lk6SQfTnJOkv1JDrfpqtY2SW5PMp3k\niSSXDr3P9tb+cJLtSzUoSdL8Rj0C+HfAH1TVXwM+CDwN3AQcqKr1wIG2DHA1sL69dgJ3ACQ5B9gF\nXA5cBuw6GRqSpPGbNwCSvBv4aeBOgKr6i6p6FdgK7G3N9gLXtvmtwF01cBBYmeQC4Cpgf1Udr6pX\ngP3A5kUdjSRpZKMcAbwPOAb8hySPJfntJD8KnF9VLwK06Xmt/WrghaHtZ1ptrrokaQJGCYAVwKXA\nHVX1IeD/8v3TPbPJLLU6Rf2NGyc7kxxKcujYsWMjdE+SdDpGCYAZYKaqHm7L9zIIhJfaqR3a9OhQ\n+7VD268Bjpyi/gZVtbuqNlTVhqmpqYWMRZK0APMGQFX9b+CFJO9vpY3AU8A+4OSdPNuB+9r8PuD6\ndjfQFcCJdoroAWBTklXt4u+mVpMkTcCKEdv9EvC7Sc4GngVuYBAe9yTZATwPXNfa3g9sAaaB11pb\nqup4kluAR1q7m6vq+KKMQpK0YCMFQFU9DmyYZdXGWdoWcOMc77MH2LOQDkqSloZPAktSpwwASeqU\nASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMjBUCS55L8cZLHkxxqtXOS7E9yuE1XtXqS3J5kOskT\nSS4dep/trf3hJNuXZkiSpFEs5AjgZ6rqkqra0JZvAg5U1XrgQFsGuBpY3147gTtgEBjALuBy4DJg\n18nQkCSN35mcAtoK7G3ze4Frh+p31cBBYGWSC4CrgP1VdbyqXgH2A5vPYP+SpDMwagAU8F+TPJpk\nZ6udX1UvArTpea2+GnhhaNuZVpurLkmagBUjtvtIVR1Jch6wP8mfnKJtZqnVKepv3HgQMDsBLrzw\nwhG7J0laqJGOAKrqSJseBb7I4Bz+S+3UDm16tDWfAdYObb4GOHKK+pv3tbuqNlTVhqmpqYWNRpI0\nsnkDIMmPJnnXyXlgE/B1YB9w8k6e7cB9bX4fcH27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJRTQOcD\nX0xysv3vVdUfJHkEuCfJDuB54LrW/n5gCzANvAbcAFBVx5PcAjzS2t1cVccXbSSSpAWZNwCq6lng\ng7PUXwY2zlIv4MY53msPsGfh3ZQkLTafBJakThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBI\nUqcMAEnq1MgBkOSsJI8l+VJbvijJw0kOJ/lCkrNb/R1tebqtXzf0Hp9u9WeSXLXYg5EkjW4hRwC/\nDDw9tPxZ4LaqWg+8Auxo9R3AK1X148BtrR1JLga2AT8JbAZ+I8lZZ9Z9SdLpGikAkqwBPg78dlsO\n8FHg3tZkL3Btm9/almnrN7b2W4G7q+o7VfVNYBq4bDEGIUlauFGPAP4t8I+Bv2zL7wVerarX2/IM\nsLrNrwZeAGjrT7T236vPss33JNmZ5FCSQ8eOHVvAUCRJCzFvACT5WeBoVT06XJ6lac2z7lTbfL9Q\ntbuqNlTVhqmpqfm6J0k6TStGaPMR4JokW4B3Au9mcESwMsmK9il/DXCktZ8B1gIzSVYA7wGOD9VP\nGt5GkjRm8x4BVNWnq2pNVa1jcBH3K1X1t4EHgU+0ZtuB+9r8vrZMW/+VqqpW39buEroIWA98bdFG\nIklakFGOAObyKeDuJL8GPAbc2ep3Ap9LMs3gk/82gKp6Msk9wFPA68CNVfXdM9i/JOkMLCgAquoh\n4KE2/yyz3MVTVd8Grptj+88An1loJyVJi88ngSWpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKRO\nGQCS1Kl5AyDJO5N8LckfJXkyyT9v9YuSPJzkcJIvJDm71d/Rlqfb+nVD7/XpVn8myVVLNShJ0vxG\nOQL4DvDRqvogcAmwOckVwGeB26pqPfAKsKO13wG8UlU/DtzW2pHkYmAb8JPAZuA3kpy1mIORJI1u\n3gCogT9vi29vrwI+Ctzb6nuBa9v81rZMW78xSVr97qr6TlV9E5gGLluUUUiSFmykawBJzkryOHAU\n2A98A3i1ql5vTWaA1W1+NfACQFt/AnjvcH2WbSRJYzZSAFTVd6vqEmANg0/tH5itWZtmjnVz1d8g\nyc4kh5IcOnbs2CjdkySdhgXdBVRVrwIPAVcAK5OsaKvWAEfa/AywFqCtfw9wfLg+yzbD+9hdVRuq\nasPU1NRCuidJWoBR7gKaSrKyzf8V4GPA08CDwCdas+3AfW1+X1umrf9KVVWrb2t3CV0ErAe+tlgD\nkSQtzIr5m3ABsLfdsfM24J6q+lKSp4C7k/wa8BhwZ2t/J/C5JNMMPvlvA6iqJ5PcAzwFvA7cWFXf\nXdzhSJJGNW8AVNUTwIdmqT/LLHfxVNW3gevmeK/PAJ9ZeDclSYvNJ4ElqVMGgCR1ygCQpE4ZAJLU\nKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0y\nACSpUwaAJHVqlP8TWAu07qYvT2S/z9368YnsV1pqk/o39VbnEYAkdWreAEiyNsmDSZ5O8mSSX271\nc5LsT3K4TVe1epLcnmQ6yRNJLh16r+2t/eEk25duWJKk+YxyBPA68KtV9QHgCuDGJBcDNwEHqmo9\ncKAtA1wNrG+vncAdMAgMYBdwOXAZsOtkaEiSxm/eAKiqF6vqD9v8/wGeBlYDW4G9rdle4No2vxW4\nqwYOAiuTXABcBeyvquNV9QqwH9i8qKORJI1sQdcAkqwDPgQ8DJxfVS/CICSA81qz1cALQ5vNtNpc\ndUnSBIwcAEl+DPiPwK9U1bdO1XSWWp2i/ub97ExyKMmhY8eOjdo9SdICjRQASd7O4Jf/71bVf2rl\nl9qpHdr0aKvPAGuHNl8DHDlF/Q2qandVbaiqDVNTUwsZiyRpAUa5CyjAncDTVfVvhlbtA07eybMd\nuG+ofn27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJQHwT4C/B3gj5M83mr/BLgVuCfJDuB54Lq27n5g\nCzANvAbcAFBVx5PcAjzS2t1cVccXZRSSpAWbNwCq6n8w+/l7gI2ztC/gxjneaw+wZyEdlCQtDZ8E\nlqROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6tQoXwctSQCsu+nLk+6CFpEB8BYyyX+cz9368YntW9Lp8RSQJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6tS8AZBkT5KjSb4+VDsnyf4kh9t0Vasnye1JppM8keTSoW22t/aHk2xfmuFIkkY1yhHA7wCb\n31S7CThQVeuBA20Z4GpgfXvtBO6AQWAAu4DLgcuAXSdDQ5I0GfM+CFZVX02y7k3lrcCVbX4v8BDw\nqVa/q6oKOJhkZZILWtv9VXUcIMl+BqHy+TMegdQZn8bVYjndJ4HPr6oXAarqxSTntfpq4IWhdjOt\nNlf9ByTZyeDogQsvvPA0u6dxm9QvJZ9Alk7fYl8Eziy1OkX9B4tVu6tqQ1VtmJqaWtTOSZK+73SP\nAF5KckH79H8BcLTVZ4C1Q+3WAEda/co31R86zX1LPxQOPvsy2zwdo2XsdANgH7AduLVN7xuq/2KS\nuxlc8D3RQuIB4F8MXfjdBHz69LstDUzq1NPd73t5IvuVFtO8AZDk8ww+vZ+bZIbB3Ty3Avck2QE8\nD1zXmt8PbAGmgdeAGwCq6niSW4BHWrubT14QliRNxih3AX1yjlUbZ2lbwI1zvM8eYM+CeidJWjI+\nCSxJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6NPQCSbE7yTJLpJDeNe/+SpIGxBkCS\ns4BfB64GLgY+meTicfZBkjQw7iOAy4Dpqnq2qv4CuBvYOuY+SJIYfwCsBl4YWp5pNUnSmK0Y8/4y\nS63e0CDZCexsi3+e5JnT3Ne5wJ+d5rbLlWMekw9/b+5nx71r8OfchXz2jMb8V0dpNO4AmAHWDi2v\nAY4MN6iq3cDuM91RkkNVteFM32c5ccx9cMx9GMeYx30K6BFgfZKLkpwNbAP2jbkPkiTGfARQVa8n\n+UXgAeAsYE9VPTnOPkiSBsZ9Coiquh+4fwy7OuPTSMuQY+6DY+7Dko85VTV/K0nSW45fBSFJnVr2\nATDfV0skeUeSL7T1DydZN/5eLq4RxvwPkzyV5IkkB5KMdEvYD7NRv0IkySeSVJJlf8fIKGNO8rfa\nz/rJJL837j4uthH+bl+Y5MEkj7W/31sm0c/FkmRPkqNJvj7H+iS5vf15PJHk0kXtQFUt2xeDC8nf\nAN4HnA38EXDxm9r8feA32/w24AuT7vcYxvwzwI+0+V/oYcyt3buArwIHgQ2T7vcYfs7rgceAVW35\nvEn3ewxj3g38Qpu/GHhu0v0+wzH/NHAp8PU51m8B/guDZ6iuAB5ezP0v9yOAUb5aYiuwt83fC2xM\nMtsDacvFvGOuqger6rW2eJDB8xbL2ahfIXIL8C+Bb4+zc0tklDH/XeDXq+oVgKo6OuY+LrZRxlzA\nu9v8e3jTc0TLTVV9FTh+iiZbgbtq4CCwMskFi7X/5R4Ao3y1xPfaVNXrwAngvWPp3dJY6Ndp7GDw\nCWI5m3fMST4ErK2qL42zY0tolJ/zTwA/keR/JjmYZPPYerc0RhnzPwN+LskMg7sJf2k8XZuYJf36\nnLHfBrrI5v1qiRHbLCcjjyfJzwEbgL+xpD1aeqccc5K3AbcBPz+uDo3BKD/nFQxOA13J4Cjvvyf5\nqap6dYn7tlRGGfMngd+pqn+d5MPA59qY/3LpuzcRS/r7a7kfAcz71RLDbZKsYHDYeKpDrh92o4yZ\nJB8D/ilwTVV9Z0x9WyrzjfldwE8BDyV5jsG50n3L/ELwqH+376uq/1dV3wSeYRAIy9UoY94B3ANQ\nVf8LeCeD7wl6qxrp3/vpWu4BMMpXS+wDtrf5TwBfqXZ1ZZmad8ztdMi/Z/DLf7mfF4Z5xlxVJ6rq\n3KpaV1XrGFz3uKaqDk2mu4tilL/b/5nBBX+SnMvglNCzY+3l4hplzM8DGwGSfIBBABwbay/Hax9w\nfbsb6ArgRFW9uFhvvqxPAdUcXy2R5GbgUFXtA+5kcJg4zeCT/7bJ9fjMjTjmfwX8GPD77Xr381V1\nzcQ6fYZGHPNbyohjfgDYlOQp4LvAP6qqlyfX6zMz4ph/FfitJP+AwamQn1/OH+iSfJ7BKbxz23WN\nXcDbAarqNxlc59gCTAOvATcs6v6X8Z+dJOkMLPdTQJKk02QASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nygCQpE4ZAJLUqf8PwRa4eRxq3rIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd9901296d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'92.98'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
